package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.WyzxModuleTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author Administrator
 * @description 描述
 * @time 2017/4/17 0017
 */
public class WyzxDownloader extends AbstractCommonDownLoader implements Cloneable{
    private Logger logger = Logger.getLogger(WyzxDownloader.class);
    String url = "http://www.wyzxwk.com/article/center/index.html";
    private Map<String,File> files=null;
    public Map<String,File> getFiles(){
        return files;
    }

    public List<Map<String,Object>> getUrlList(){

        HashMap<String,Object> classMap = new HashMap<String,Object>();
        classMap.put("时代观察",14);
        classMap.put("舆论战争",19);
        classMap.put("经济视点",15);
        classMap.put("社会民生",17);
        classMap.put("三农关注",24);
        classMap.put("产业研究",74);
        classMap.put("国际纵横",16);
        classMap.put("国防外交",18);
        classMap.put("理想之旅",12);
        classMap.put("思潮碰撞",13);
        classMap.put("文艺新生",23);
        classMap.put("读书交流",31);
        classMap.put("历史视野",21);
        classMap.put("中华文化",22);
        classMap.put("中华医药",26);
        classMap.put("共产党人",29);
        classMap.put("风华正茂",20);
        classMap.put("工农之声",25);
        classMap.put("网友杂谈",27);
        classMap.put("网友时评",28);
        classMap.put("乌有公告",30);
        classMap.put("红色旅游",32);
        classMap.put("乌有讲堂",61);
        classMap.put("书画欣赏",108);


        List<Map<String,Object>> urlList = new ArrayList<Map<String,Object>>();
        Document document = null;
        long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("Wyzx.URLWaitTime"));
        long remianing = Long.valueOf(PropertiesUtil.getVal("Wyzx.URLWaitTime"));
        boolean flag = true;
        while(flag&&remianing>=0) {
            try {
                remianing = future - System.currentTimeMillis();
                try {
                    document = get(url);
                }catch (Exception e){
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    document = get(url, true);
                }


                Elements sub_forumList = document.select(".g-bd").select(".g-sd").select(".g-sd-nav").select("ul");

                for(int i = 0;i<sub_forumList.size();i++){
                    Elements liList = sub_forumList.get(i).select("li");
                    for(int j = 0;j<liList.size();j++){
                        String url = liList.get(j).select("a").get(0).attr("href");
                        String name = liList.get(j).text();

                        Map<String,Object> map = new HashMap<String,Object>();
                        map.put("url",url);
                        map.put("channel",name);
                        map.put("classId",classMap.get(name));
                        System.out.println(name+":"+url+":"+classMap.get(name));
                        urlList.add(map);
                    }
                }


                flag = false;
            }catch (Exception e){
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
        }

        return urlList;
    }


    public void generateTask(List<Map<String, Object>> urlList){
        for(int i = 0;i<urlList.size();i++){

            int page = 1;
            String sub_forum_url = urlList.get(i).get("url").toString();
            String channel = urlList.get(i).get("channel").toString();
            String classId = urlList.get(i).get("classId").toString();

            try {
                WyzxModuleTask task = new WyzxModuleTask(channel,sub_forum_url,classId);
                CommonTaskManager.add("WyzxModule", task);
            }catch (Exception e){
                logger.error(e.getMessage(),e);
            }



           /* try {
                Document document = null;
                try {
                    document = get(sub_forum_url);
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                    document = get(sub_forum_url,ProxyPool.proxy());
                }

                Elements pageElements = document.select(".g-bd").select(".g-mn");

                Elements content_urls = pageElements.select(".g-mnc").select(".m-pt");

                for(int j = 0;j<content_urls.size();j++){
                    String content_url = content_urls.get(j).select("a").get(0).attr("href");
                    WyzxTask task = new WyzxTask(URLDecoder.decode(content_url, "UTF-8"));
                    CommonTaskManager.addTask("Wyzx", task,1000);
                    System.out.println(content_url);
                }

                String pageInfo = pageElements.select(".m-pages").select("a").get(pageElements.select(".m-pages").select("a").size()-2).text();

                if(pageInfo!=null&&!"".equals(pageInfo)) {
                    page = Integer.parseInt(pageInfo);
                }

                if(page>=2){
                    for(int m = 2;m<=page;m++){
                        String new_url = sub_forum_url+"/index_"+m+".html";
                        Document document1 = null;

                        try {
                            document1 = get(new_url);
                        } catch (Exception e) {
                            logger.error(e.getMessage(), e);
                            document1 = get(new_url,ProxyPool.proxy());
                        }

                        Elements pageElements1 = document1.select(".g-bd").select(".g-mn");

                        Elements content_urls1 = pageElements1.select(".g-mnc").select(".m-pt");

                        for(int j = 0;j<content_urls1.size();j++){
                            String content_url = content_urls1.get(j).select("a").get(0).attr("href");
                            WyzxTask task = new WyzxTask(URLDecoder.decode(content_url, "UTF-8"));
                            CommonTaskManager.addTask("Wyzx", task,1000);
                            System.out.println(content_url);
                        }

                    }
                }

            }catch (Exception e){
                logger.error(e.getMessage(), e);
                e.printStackTrace();

            }*/
        }
    }

    public JSONObject crawlItem(String content_url, String channel, String classId){
        files=new ConcurrentHashMap<>();
        JSONObject item=new JSONObject();
        JSONObject data = new JSONObject();

        int page = 1;
        try {
            Document document = null;
            try {
                Page pageContent = null;
                try {
                    pageContent = getHtml(content_url);

                    if (pageContent == null && pageContent.getStatusCode() != 200) {
                        pageContent = getHtml(content_url, true);
                    }
                }catch (Exception e){
                    logger.error(e.getMessage(),e);
                    pageContent = getHtml(content_url, true);
                    e.printStackTrace();
                }

                if (null == pageContent) {

                } else if (pageContent.getStatusCode() == 0) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                } else if (pageContent.getStatusCode() == 200) {
                    try {
                        document = pageContent.getHtml().getDocument();

                        String idInfo = "";
                        String idRegex = "[0-9]{1,}\\.html";
                        Pattern p = Pattern.compile(idRegex);
                        Matcher m = p.matcher(content_url);
                        while (m.find()) {
                            idInfo = m.group();
                        }

                        String id = "";
                        if (idInfo != null && !"".equals(idInfo)) {
                            id = idInfo.split("\\.")[0];
                        } else {
                            System.out.println("id为空");
                        }


                        String title = document.title();



                        Elements pageElements = document.select(".g-bd").select(".g-mn-1");

                        String content = pageElements.select(".m-article").select("article").select("p").text();
                        JSONArray img_urls=new JSONArray();
                        Elements imgElements=pageElements.select(".m-article").select("article").select("p").select("img");
                        for(int imgIndex=0;imgIndex<imgElements.size();imgIndex++){
                            Element imgElement=imgElements.get(imgIndex);
                            String img_url=imgElement.absUrl("src");
                            img_urls.add(img_url);
                            System.out.println(img_url);
                        }
                        if(content == null||"".equals(content)){
                            content = pageElements.select(".m-article").select("article").text();
                        }

                        String timeInfo = "";
                        String publish_time = "";

                        if(content==null||"".equals(content)) {
                            pageElements = document.select("#main").select(".main_left").select(".content");
                            content = pageElements.select("#text_content").text();

                            try{
                                timeInfo = pageElements.select(".property").select("span").get(1).text();
                            }catch (Exception e){
                                logger.error(e.getMessage(),e);
                                e.printStackTrace();
                            }

                            String timeRegex = "[0-9]{1,}.*";

                            Pattern p1 = Pattern.compile(timeRegex);
                            Matcher m1 = p1.matcher(timeInfo);

                            while (m1.find()) {
                                publish_time = m1.group()+" 00:00:00";
                            }

                        }else {

                            try {
                                timeInfo = pageElements.select(".m-article").select(".f-fl").select(".s-grey-3").get(2).text();
                            } catch (Exception e) {
                                logger.error(e.getMessage(),e);
                                e.printStackTrace();
                            }

                        /*String timeRegex1 = "[0-9]{1,}.*";

                        Pattern p1 = Pattern.compile(timeRegex1);
                        Matcher m1 = p1.matcher(timeInfo);
                        while (m1.find()) {
                            publish_time = m1.group() + " 00:00:00";
                            publish_time.replaceAll("\\·","");
                        }
                        */


                            if (timeInfo != null && !"".equals(timeInfo)) {
                                publish_time = timeInfo + " 00:00:00";
                            }
                        }

                        String spider_time = DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss");

                        data.put("id", id);
                        data.put("content", content);
                        data.put("img_urls",img_urls);
                        data.put("url", content_url);
                        data.put("title", title);
                        data.put("publish_time", publish_time);
                        data.put("spider_time", spider_time);

                        long tsp= DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime();
                        String filePrefix="Wyzx";//文件前缀
                        String fileName=filePrefix+"_"+tsp+"_"+"_"+id;
                        String publishDate=DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                        String speechesId="Wyzx_speeches_"+publishDate+"_"+fileName;
                        File file= FileUtils.createTempFile(speechesId+"_pageNo_1",document.html().toString());
                        files.put(speechesId+"_pageNo_1",file);
                        //帖子首页的文件

                        int reply_count = 0;

                        JSONArray reply_nodes = new JSONArray();
                        List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                        if (id != null && !"".equals(id)) {
                            String reply_url = "http://www.wyzxwk.com/e/pl/list.php?id=" + id + "&classid=" + classId;

                            Document reply_document = null;
                            Page reply_pageContent = null;

                            try {
                                reply_pageContent = getHtml(reply_url);
                                if (reply_pageContent == null && reply_pageContent.getStatusCode() != 200) {
                                    reply_pageContent = getHtml(reply_url, true);
                                }
                            } catch (Exception e) {
                                logger.error(e.getMessage(),e);
                                reply_pageContent = getHtml(reply_url, true);
                                e.printStackTrace();
                            }

                            if (null == reply_pageContent) {

                            } else if (reply_pageContent.getStatusCode() == 0) {

                            } else if (String.valueOf(reply_pageContent.getStatusCode()).startsWith("1")) {

                            } else if (reply_pageContent.getStatusCode() == 200) {


                                reply_document = reply_pageContent.getHtml().getDocument();

                                String replyInfo = reply_document.select(".m-plhd").select(".f-fl").text();

                                String replyRegex = "[0-9]{1,}";

                                Pattern p1 = Pattern.compile(replyRegex);
                                Matcher m1 = p1.matcher(replyInfo);
                                while (m1.find()) {
                                    reply_count = Integer.parseInt(m1.group());
                                }

                                data.put("reply_count", reply_count);


                                /**
                                 * 爬取评论信息
                                 */
                                if (reply_count > 0) {
                                    if ("加载更多评论".equals(reply_document.select("#j-plmore").text())) {
                                        String new_reply_url = "http://www.wyzxwk.com/e/pl/doaction.php?id=" + id + "&classid=" + classId + "&order=0&pgcrt=1&ismore=0&enews=changePage";
                                        Document new_reply_document = null;
                                        Page pageContent1 = null;
                                        try {
                                            pageContent1 = getHtml(new_reply_url);

                                            if (pageContent1 == null && pageContent1.getStatusCode() != 200) {
                                                pageContent1 = getHtml(new_reply_url, true);
                                            }
                                        }catch (Exception e){
                                            logger.error(e.getMessage(),e);
                                            pageContent1 = getHtml(new_reply_url, true);
                                            e.printStackTrace();
                                        }

                                        int total_page = 1;

                                        if (null == pageContent1) {

                                        } else if (pageContent1.getStatusCode() == 0) {

                                        } else if (String.valueOf(pageContent1.getStatusCode()).startsWith("1")) {

                                        } else if (pageContent1.getStatusCode() == 200) {
                                            new_reply_document = pageContent1.getHtml().getDocument();
                                            String pageInfo = new_reply_document.select(".m-pages").text();

                                            pageInfo = pageInfo.replaceAll(" ","");
                                            if(pageInfo!=null&&!"".equals(pageInfo)){
                                                if(pageInfo.length()>1){
                                                    total_page = Integer.parseInt(pageInfo.substring(pageInfo.length()-2,pageInfo.length()-1));
                                                }
                                            }

                                            Elements replyList = new_reply_document.select("section");

                                            for (int i = 0; i < replyList.size(); i++) {
                                                try {
                                                    JSONObject reply_object = new JSONObject();

                                                    Elements imgElments = replyList.get(i).select("img");
                                                    String publish_user_photo = "";
                                                    if (imgElments.size() > 0 && !"".equals(imgElments) && imgElments != null) {
                                                        publish_user_photo = imgElments.get(0).attr("href");
                                                    }
                                                    String publish_user = replyList.get(i).select(".u-uname").get(0).text();

                                                    String reply_publish_time = "";
                                                    if(replyList.get(i).select(".s-grey-4").size()>1) {
                                                        reply_publish_time = replyList.get(i).select(".s-grey-4").get(0).text();
                                                    }else{
                                                        reply_publish_time = replyList.get(i).select(".s-grey-4").text();
                                                    }

                                                    String reply_id = replyList.get(i).select("[id^=j-repl-]").get(0).attr("data-plid");
                                                    String parent_id = replyList.get(i).select("[id^=j-repl-]").get(0).attr("data-parent");

                                                    //ystem.out.println(replyList.get(i).select("span").get(0));

                                                    String reply_content = replyList.get(i).select(".u-st").text();
                                                    JSONArray reply_img_urls=new JSONArray();
                                                    Elements replyImgElements=replyList.get(i).select(".u-st").select("img");
                                                    for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                                        Element imgElement=replyImgElements.get(imgIndex);
                                                        String img_url=imgElement.absUrl("src");
                                                        reply_img_urls.add(img_url);
                                                    }
                                                    if (parent_id.equals(reply_id)) {
                                                        parent_id = id;
                                                    }

                                                    int like_count = 0;
                                                    int dislike_count = 0;

                                                    String likeInfo = replyList.get(i).select("[id^=j-zc-]").get(0).text();
                                                    String dislikeInfo = replyList.get(i).select("[id^=j-fd-]").get(0).text();

                                                    String likeRegex = "[0-9]{1,}";

                                                    Pattern p2 = Pattern.compile(likeRegex);
                                                    Matcher m2 = p2.matcher(likeInfo);

                                                    Matcher m3 = p2.matcher(dislikeInfo);
                                                    while (m2.find()) {
                                                        like_count = Integer.parseInt(m2.group());
                                                    }
                                                    while (m3.find()) {
                                                        dislike_count = Integer.parseInt(m3.group());
                                                    }

                                                    reply_object.put("publish_user_photo", publish_user_photo);
                                                    reply_object.put("publish_user", publish_user);
                                                    reply_object.put("publish_time", reply_publish_time+":00");
                                                    reply_object.put("id", reply_id);
                                                    reply_object.put("publish_user_id", reply_id);
                                                    reply_object.put("ancestor_id", id);
                                                    reply_object.put("parent_id", parent_id);
                                                    reply_object.put("like_count", like_count);
                                                    reply_object.put("dislike_count", dislike_count);
                                                    reply_object.put("content", reply_content);
                                                    reply_object.put("img_urls",reply_img_urls);
                                                    reply_object.put("spider_time", DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss"));

                                                    reply_nodes.add(reply_object);

                                                } catch (Exception e) {
                                                    logger.error(e.getMessage(),e);
                                                    e.printStackTrace();
                                                    continue;
                                                }

                                            }

                                            JSONArray replys = new JSONArray();
                                            if(total_page>=2){
                                                //for(int j = 0;j<=2;j++){
                                                    int threadNum = Integer.valueOf(PropertiesUtil.getVal("Wyzx.ReplyThreadCount"));
                                                    ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                                                    try {
                                                        List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                                        ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, total_page, threadNum);
                                                        for (int i = 0; i < splitPages.size(); i++) {
                                                            WyzxDownloader downLoader = this.cloneDownLoader();
                                                            WyzxDownloader.ReplyCrawler callable = new WyzxDownloader.ReplyCrawler(downLoader, splitPages.get(i), id, channel,classId,speechesId);
                                                            FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                                            futureTasks.add(futureTask);
                                                            executorService.submit(futureTask);
                                                        }
                                                        for (FutureTask<JSONArray> futureTask : futureTasks) {
                                                            JSONArray replyArrays = futureTask.get();
                                                            for (int i = 0; i < replyArrays.size(); i++) {
                                                                JSONObject object = replyArrays.getJSONObject(i);
                                                                replys.add(object);
                                                            }
                                                        }
                                                        if (replys.size() > 0 && replys != null) {
                                                            for (int i = 0; i < replys.size(); i++) {
                                                                JSONObject reply_object1 = replys.getJSONObject(i);
                                                                reply_nodes.add(reply_object1);
                                                            }
                                                        }
                                                        if (reply_nodes.size() > 0 && reply_nodes != null) {
                                                            for (int i = 0; i < reply_nodes.size(); i++) {
                                                                JSONObject reply_object2 = reply_nodes.getJSONObject(i);
                                                                Map<String, Object> map = reply_object2;
                                                                list.add(map);
                                                            }
                                                        }
                                                        Collections.sort(list, new Comparator<Map<String, Object>>() {

                                                            public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                                                String s1 = o1.get("publish_time").toString();
                                                                String s2 = o2.get("publish_time").toString();
                                                                int flag = s2.compareTo(s1);
                                                                return flag;
                                                            }
                                                        });

                                                    }catch (Exception e){
                                                        e.printStackTrace();
                                                        logger.error(e.getMessage(), e);
                                                    } finally {
                                                        executorService.shutdown();
                                                    }
                                                //}
                                            }else{
                                                if (reply_nodes.size() > 0 && reply_nodes != null) {
                                                    for (int i = 0; i < reply_nodes.size(); i++) {
                                                        JSONObject new_reply_object = reply_nodes.getJSONObject(i);
                                                        Map<String, Object> map = new_reply_object;
                                                        list.add(map);
                                                    }
                                                }
                                            }

                                        }else{

                                        }


                                    } else {
                                        Elements replyList = reply_document.select("section");
                                        for (int i = 0; i < replyList.size(); i++) {
                                            try {

                                                JSONObject reply_object = new JSONObject();

                                                Elements imgElments = replyList.get(i).select("img");
                                                String publish_user_photo = "";
                                                if (imgElments.size() > 0 && !"".equals(imgElments) && imgElments != null) {
                                                    publish_user_photo = imgElments.get(0).attr("href");
                                                }
                                                String publish_user = replyList.get(i).select(".u-uname").get(0).text();

                                                String reply_publish_time = replyList.get(i).select(".s-grey-4").text();

                                                String reply_id = replyList.get(i).select("[id^=j-repl-]").get(0).attr("data-plid");
                                                String parent_id = replyList.get(i).select("[id^=j-repl-]").get(0).attr("data-parent");

                                                //ystem.out.println(replyList.get(i).select("span").get(0));

                                                String reply_content = replyList.get(i).select(".u-st").text();
                                                if (parent_id.equals(reply_id)) {
                                                    parent_id = id;
                                                }

                                                int like_count = 0;
                                                int dislike_count = 0;

                                                String likeInfo = replyList.get(i).select("[id^=j-zc-]").get(0).text();
                                                String dislikeInfo = replyList.get(i).select("[id^=j-fd-]").get(0).text();

                                                String likeRegex = "[0-9]{1,}";

                                                Pattern p2 = Pattern.compile(likeRegex);
                                                Matcher m2 = p2.matcher(likeInfo);

                                                Matcher m3 = p2.matcher(dislikeInfo);
                                                while (m2.find()) {
                                                    like_count = Integer.parseInt(m2.group());
                                                }
                                                while (m3.find()) {
                                                    dislike_count = Integer.parseInt(m3.group());
                                                }

                                                reply_object.put("publish_user_photo", publish_user_photo);
                                                reply_object.put("publish_user", publish_user);
                                                reply_object.put("publish_time", reply_publish_time+":00");
                                                reply_object.put("id", reply_id);
                                                reply_object.put("publish_user_id", reply_id);
                                                reply_object.put("ancestor_id", id);
                                                reply_object.put("parent_id", parent_id);
                                                reply_object.put("like_count", like_count);
                                                reply_object.put("dislike_count", dislike_count);
                                                reply_object.put("content", reply_content);
                                                reply_object.put("spider_time", DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss"));

                                                reply_nodes.add(reply_object);

                                            } catch (Exception e) {
                                                logger.error(e.getMessage(),e);
                                                e.printStackTrace();
                                                continue;
                                            }

                                        }
                                        if (reply_nodes.size() > 0 && reply_nodes != null) {
                                            for (int i = 0; i < reply_nodes.size(); i++) {
                                                JSONObject new_reply_object = reply_nodes.getJSONObject(i);
                                                Map<String, Object> map = new_reply_object;
                                                list.add(map);
                                            }
                                        }
                                    }
                                }

                            } else {

                            }
                            data.put("reply_nodes",list);
                        }


                        //评论:http://www.wyzxwk.com/e/pl/doaction.php?id=372683&classid=14&order=0&pgcrt=2&ismore=0&enews=changePage

                    } catch (Exception e) {
                        logger.error(e.getMessage(), e);
                        e.printStackTrace();
                    }

                }else{

                }
            }catch (Exception e){
                logger.error(e.getMessage(),e);
                e.printStackTrace();
            }
        }catch (Exception e){
            item = new JSONObject();
            data = new JSONObject();
            logger.error(e.getMessage(),e);
        }
        item.put("data", data);
        return item;

    }

    public WyzxDownloader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        WyzxDownloader loader = new WyzxDownloader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public static void main(String[] args) {
        WyzxDownloader downloader = new WyzxDownloader();
        //List<Map<String,Object>> urlList = downloader.getUrlList();
        //downloader.generateTask(urlList);

        downloader.crawlItem("http://www.wyzxwk.com/Article/yulun/2017/01/375947.html","舆论战争","19");
    }

    public class ReplyCrawler implements Callable<JSONArray> {
        private WyzxDownloader downloader;
        private ArrayList<Integer> pages;
        private String channel;
        private String ancestor_id;
        private String classId;
        private String speeches_id;


        @Override
        public JSONArray call() throws Exception {
            JSONArray array =new JSONArray();

            for(int k=0;k<pages.size();k++){

                String new_url = "http://www.wyzxwk.com/e/pl/doaction.php?id=" + ancestor_id+ "&classid=" + classId + "&order=0&pgcrt="+pages.get(k)+"&ismore=0&enews=changePage";
                Document document = null;
                Page pageContent = null;

                try {
                    pageContent = downloader.getHtml(new_url);

                    if (pageContent == null && pageContent.getStatusCode() != 200) {
                        pageContent = downloader.getHtml(new_url, true);
                    }
                }catch (Exception e){
                    logger.error(e.getMessage(),e);
                    pageContent = downloader.getHtml(new_url, true);
                    e.printStackTrace();
                }

                if (null == pageContent) {

                } else if (pageContent.getStatusCode() == 0) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                } else if (pageContent.getStatusCode() == 200) {
                        document  = pageContent.getHtml().getDocument();
                        File file=FileUtils.createTempFile(speeches_id+"_pageNo_"+pages.get(k),document.html().toString());
                        files.put(speeches_id+"_pageNo_"+pages.get(k),file);
                        Elements replyList = document.select("section");
                        for (int i = 0; i < replyList.size(); i++) {
                            try {
                                JSONObject reply_object = new JSONObject();
                                Elements imgElments = replyList.get(i).select("img");
                                String publish_user_photo = "";
                                if (imgElments.size() > 0 && !"".equals(imgElments) && imgElments != null) {
                                    publish_user_photo = imgElments.get(0).attr("href");
                                }
                                String publish_user = replyList.get(i).select(".u-uname").get(0).text();

                                String reply_publish_time = replyList.get(i).select(".s-grey-4").text();

                                String reply_id = replyList.get(i).select("[id^=j-repl-]").get(0).attr("data-plid");
                                String parent_id = replyList.get(i).select("[id^=j-repl-]").get(0).attr("data-parent");

                                //ystem.out.println(replyList.get(i).select("span").get(0));

                                String reply_content = replyList.get(i).select(".u-st").text();
                                JSONArray reply_img_urls=new JSONArray();
                                Elements replyImgElements=replyList.get(i).select(".u-st").select("img");
                                for(int imgIndex=0;imgIndex<replyImgElements.size();imgIndex++){
                                    Element imgElement=replyImgElements.get(imgIndex);
                                    String img_url=imgElement.absUrl("src");
                                    reply_img_urls.add(img_url);
                                }
                                if (parent_id.equals(reply_id)) {
                                    parent_id = ancestor_id;
                                }

                                int like_count = 0;
                                int dislike_count = 0;

                                String likeInfo = replyList.get(i).select("[id^=j-zc-]").get(0).text();
                                String dislikeInfo = replyList.get(i).select("[id^=j-fd-]").get(0).text();

                                String likeRegex = "[0-9]{1,}";

                                Pattern p2 = Pattern.compile(likeRegex);
                                Matcher m2 = p2.matcher(likeInfo);

                                Matcher m3 = p2.matcher(dislikeInfo);
                                while (m2.find()) {
                                    like_count = Integer.parseInt(m2.group());
                                }
                                while (m3.find()) {
                                    dislike_count = Integer.parseInt(m3.group());
                                }

                                reply_object.put("publish_user_photo", publish_user_photo);
                                reply_object.put("publish_user", publish_user);
                                reply_object.put("publish_time", reply_publish_time+":00");
                                reply_object.put("id", reply_id);
                                reply_object.put("publish_user_id", reply_id);
                                reply_object.put("ancestor_id", ancestor_id);
                                reply_object.put("parent_id", parent_id);
                                reply_object.put("like_count", like_count);
                                reply_object.put("dislike_count", dislike_count);
                                reply_object.put("content", reply_content);
                                reply_object.put("img_urls",reply_img_urls);
                                reply_object.put("spider_time", DateUtils.dateToString(new Date(), "yyyy-MM-dd HH:mm:ss"));

                                array.add(reply_object);

                            } catch (Exception e){
                                logger.error(e.getMessage(),e);
                                e.printStackTrace();
                                continue;
                            }
                    }
                }else{

                }
            }



            return array;
        }

        public ReplyCrawler(WyzxDownloader downloader, ArrayList<Integer> pages, String ancestor_id, String channel, String classId,String speeches_id){
            this.downloader = downloader;
            this.pages = pages;
            this.ancestor_id = ancestor_id;
            this.classId = classId;
            this.channel = channel;
            this.speeches_id=speeches_id;

        }



    }
}
